package cn.itcast.jobs.task;

import cn.itcast.jobs.pojo.JobInfo;
import org.jsoup.Jsoup;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

@Component
public class JobProcessor implements PageProcessor {

    private String url = "https://search.51job.com/list/000000,000000,0121%252c0122%252c7201%252c7202%252c7701,32%252c01%252c38,9,99,+,2,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";


    @Override
    public void process(Page page) {
//        解析页面地址
        List<Selectable> list = page.getHtml().css("div.j_joblist div.e").nodes();
//        判断获取到的集合是否为空，如果为空，表示这是招聘详情页

        if (list.size() == 0) {
            this.saveJobInfo(page);

        } else {//        如果不为空表示列表页解析想请个的url的地址，放到任务队列中

            for (Selectable selectable : list) {
//                获取url地址
                String jobInfoUrl = selectable.links().toString();
                page.addTargetRequest(jobInfoUrl);
            }

//            获取下一页的url
            String bkUrl = page.getHtml().css("div.p_in ul li.next").nodes().toString();


//            把url放到任务中
            page.addTargetRequest(bkUrl);
        }


        String html = page.getHtml().toString();


    }

    //    解析页面分析详情页
    private void saveJobInfo(Page page) {
//创建招聘详情对象
        JobInfo jobInfo = new JobInfo();

//        解析页面封装对象
        Html html = page.getHtml();


        jobInfo.setCompanyName(html.css("div.e div.er a","text").toString());

        jobInfo.setCompanyInfo(Jsoup.parse(html.css("p.info span").nodes().toString()).text());
        jobInfo.setCompanyAddr(Jsoup.parse(html.css("p.info span").nodes().get(1).toString()).text());
//
//        jobInfo.setJobName();
//        jobInfo.setJobAddr();
//        jobInfo.setJobInfo();
//
//        jobInfo.setUrl();
//        jobInfo.setSalaryMin();
//        jobInfo.setSalaryMax();
//
//        jobInfo.setTime();

//        保存结果,此时还在内存中
        page.putField("jobInfo",jobInfo);

    }

    private Site site = Site.me()
            .setCharset("gbk")  //设置编码，大多数页面不是utf8就是gbk需要根据地址页面看
            .setTimeOut(10 * 1000)   //设置超时时间
            .setRetrySleepTime(3000)  //设置超时间隔
            .setRetryTimes(3);  //设置重试次数

    @Override
    public Site getSite() {
        return site;
    }

    @Autowired
    private SpringDataPipeline springDataPipeline;
    @Scheduled(initialDelay = 1000, fixedDelay = 100 * 1000)  //定时任务,initialDelay等待多久执行方法fixedDelay每隔多久执行方法
    public void process() {
        Spider.create(new JobProcessor())
                .addUrl(url)
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(10)
                .addPipeline(this.springDataPipeline)
                .run();

    }
}
